{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ablation study example\n",
    "\n",
    "> Imagenette/woof training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from fastcore.all import *\n",
    "from nbdev.showdoc import *\n",
    "from IPython.lib.display import FileLink"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('~/git/fastai2/nbs/examples/exps').expanduser()\n",
    "path_out = path/'out'\n",
    "path_scr = path/'complete'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fn_name(fn): return fn.name.split('.')[0]\n",
    "\n",
    "hdrs = 'name epoch trn_loss val_loss top1 top5 time'.split()\n",
    "\n",
    "def scr_results(fn):\n",
    "    return L([dict(zip(hdrs, [fn_name(fn)] + o.strip().split()))\n",
    "           for o in fn.readlines() if re.match(r'\\d+   ', o)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>epoch</th>\n",
       "      <th>trn_loss</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>top1</th>\n",
       "      <th>top5</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>206a</td>\n",
       "      <td>0</td>\n",
       "      <td>1.660802</td>\n",
       "      <td>1.637859</td>\n",
       "      <td>0.540892</td>\n",
       "      <td>0.913885</td>\n",
       "      <td>00:34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>206a</td>\n",
       "      <td>1</td>\n",
       "      <td>1.448628</td>\n",
       "      <td>1.292243</td>\n",
       "      <td>0.680510</td>\n",
       "      <td>0.966624</td>\n",
       "      <td>00:30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>206a</td>\n",
       "      <td>2</td>\n",
       "      <td>1.344256</td>\n",
       "      <td>1.319426</td>\n",
       "      <td>0.682293</td>\n",
       "      <td>0.961274</td>\n",
       "      <td>00:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>206a</td>\n",
       "      <td>3</td>\n",
       "      <td>1.285479</td>\n",
       "      <td>1.036926</td>\n",
       "      <td>0.786497</td>\n",
       "      <td>0.977325</td>\n",
       "      <td>00:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>206a</td>\n",
       "      <td>4</td>\n",
       "      <td>1.227500</td>\n",
       "      <td>1.057364</td>\n",
       "      <td>0.783694</td>\n",
       "      <td>0.976815</td>\n",
       "      <td>00:31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name  epoch  trn_loss  val_loss      top1      top5   time\n",
       "0  206a      0  1.660802  1.637859  0.540892  0.913885  00:34\n",
       "1  206a      1  1.448628  1.292243  0.680510  0.966624  00:30\n",
       "2  206a      2  1.344256  1.319426  0.682293  0.961274  00:31\n",
       "3  206a      3  1.285479  1.036926  0.786497  0.977325  00:31\n",
       "4  206a      4  1.227500  1.057364  0.783694  0.976815  00:31"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_results = L([scr_results(o) for o in path_out.glob('*.stdout')]).concat()\n",
    "df_results = DataFrame.from_records(all_results).query('time==time').astype(\n",
    "    dict(epoch=int, trn_loss=float, val_loss=float, top5=float, top1=float))\n",
    "\n",
    "df_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scr_params(fn):\n",
    "    return merge(dict([o.strip().split() for o in fn.read_text().split('--')[1:]]),\n",
    "                 {'name':fn_name(fn)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>woof</th>\n",
       "      <th>lr</th>\n",
       "      <th>size</th>\n",
       "      <th>sqrmom</th>\n",
       "      <th>mom</th>\n",
       "      <th>eps</th>\n",
       "      <th>epochs</th>\n",
       "      <th>bs</th>\n",
       "      <th>opt</th>\n",
       "      <th>sa</th>\n",
       "      <th>fp16</th>\n",
       "      <th>arch</th>\n",
       "      <th>name</th>\n",
       "      <th>pool</th>\n",
       "      <th>meta</th>\n",
       "      <th>runs</th>\n",
       "      <th>mixup</th>\n",
       "      <th>sh</th>\n",
       "      <th>sym</th>\n",
       "      <th>beta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>8e-3</td>\n",
       "      <td>192</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1e-6</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>ranger</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>xse_resnext18</td>\n",
       "      <td>02a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>8e-3</td>\n",
       "      <td>128</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1e-6</td>\n",
       "      <td>20</td>\n",
       "      <td>64</td>\n",
       "      <td>ranger</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>xse_resnext50</td>\n",
       "      <td>xf104a</td>\n",
       "      <td>MaxPool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>8e-3</td>\n",
       "      <td>256</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1e-6</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>ranger</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>xse_resnext18</td>\n",
       "      <td>03a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>8e-3</td>\n",
       "      <td>128</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1e-6</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>ranger</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>xse_resnext50</td>\n",
       "      <td>x01b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>8e-3</td>\n",
       "      <td>192</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1e-6</td>\n",
       "      <td>20</td>\n",
       "      <td>64</td>\n",
       "      <td>ranger</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>xse_resnext50</td>\n",
       "      <td>x05a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  woof    lr size sqrmom   mom   eps  epochs  bs     opt sa fp16  \\\n",
       "0    0  8e-3  192   0.99  0.95  1e-6       5  64  ranger  1    0   \n",
       "1    0  8e-3  128   0.99  0.95  1e-6      20  64  ranger  1    1   \n",
       "2    0  8e-3  256   0.99  0.95  1e-6       5  64  ranger  1    0   \n",
       "3    1  8e-3  128   0.99  0.95  1e-6       5  64  ranger  1    1   \n",
       "4    0  8e-3  192   0.99  0.95  1e-6      20  64  ranger  1    1   \n",
       "\n",
       "            arch    name     pool meta runs mixup   sh  sym beta  \n",
       "0  xse_resnext18     02a      NaN  NaN  NaN   NaN  NaN  NaN  NaN  \n",
       "1  xse_resnext50  xf104a  MaxPool  NaN  NaN   NaN  NaN  NaN  NaN  \n",
       "2  xse_resnext18     03a      NaN  NaN  NaN   NaN  NaN  NaN  NaN  \n",
       "3  xse_resnext50    x01b      NaN  NaN  NaN   NaN  NaN  NaN  NaN  \n",
       "4  xse_resnext50    x05a      NaN  NaN  NaN   NaN  NaN  NaN  NaN  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_params = DataFrame([scr_params(o) for o in path_scr.ls()]).astype({'epochs':int})\n",
    "df_params.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merge = pd.merge(df_params, df_results, 'left', on='name')\n",
    "assert df_merge[pd.isna(df_merge.arch)].empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merge.fillna(0.0, inplace=True)\n",
    "df_merge['repoch'] = df_merge.epochs - df_merge.epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='imagenette.csv' target='_blank'>imagenette.csv</a><br>"
      ],
      "text/plain": [
       "/home/jhoward/git/fastgpu/imagenette.csv"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merge.to_csv('imagenette.csv', index=False)\n",
    "FileLink('imagenette.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
